#!/bin/bash
#########################################################
# SCRIPT  : epg-swiss.sh                                #
#########################################################
# AUTHORS : Hans Weber                                  #
#           Thomas Oeding                               #
# EMAIL   : linuxluemmel.ch@gmail.com                   #
#           thomas.oeding@gmail.com                     #
# OS      : Linux/Unix                                  #
#                                                       #
# TESTED  : - Ubuntu 10.04 LTS                          #
#           - Debian 6.0                                #
#           - Snow Leopard /with macports.org installed #
#                                                       #
# TASKS   : create xml-files from tv.teleboy.ch         #
# VERSION : 0.9O                                        #
# DATE    : 11-09-10                                    #
# STATE   : RC 5                                        #
# LICENCE : GPL 3.0                                     #
#########################################################
#                                                       #
# Project-Homepage :                                    #
# http://code.google.com/p/epg-swiss                    #
#                                                       #
#########################################################



#################### Contants ###########################
nargs=2
compare=1
MAXDAYS=12
SCRIPTDIR=$(pwd)
SYSTEM=$(uname)

REQUIRED_TOOLS=`cat << EOF
rm
cat
tac
gawk
date
wc
ps
lynx
snarf
tee
sleep
head
cut
rev
EOF`

if [ "$SYSTEM" = "Darwin" ]; then
   REQUIRED_TOOLS_DARWIN=`cat << EOF
   gtee
   grm
   gcat
   gtac
   gdate
   gsed
   gtr
   gcut
   gwc
   gexpr
   lynx
   snarf
EOF`

fi

ZERO_ERRORS=0
ERROR_TOOLNOTFOUND=1
ERROR_WRONG_ARG=2
ERROR_AWRONG=3
ERROR_HOST=4
ERROR_TOOLCHAIN_TO_OLD=5
ERROR_INDEX_TO_HIGH=6
ERROR_FAST_MODE=7
ERRRO_EDIT_CONFIGURATION=8

# In the case of a error the verbose mode may help

VERBOSE=""
SHOWINDEX=""


# If this variable is set to "TRUE" then icon-url's for the channels
# is included in the produced xml
# By default this value is set to "TRUE"
# If you do not wish to use channel-icons set the variable 
# INCLUDE_ICONS to empty like INCLUDE_ICONS="".

INCLUDE_ICONS="TRUE"

#########################################################



#########################################################
#             Test Installation                         #
#########################################################
GNU_TEE=tee
GNU_RM=rm
GNU_CAT=cat
GNU_TAC=tac
GNU_WC=wc
GNU_TR=tr
GNU_DATE=date
GNU_CUT=cut
GNU_SED=sed
GNU_AWK=gawk
GNU_EXPR=expr

if [ "$SYSTEM" = "Darwin" ]; then
   ### snow leopard #####################################

   for REQUIRED_TOOLS_DARWIN in ${REQUIRED_TOOLS_DARWIN}
   do
      which ${REQUIRED_TOOLS_DARWIN} >/dev/null 2>&1
      if [ $? -eq 1 ]; then
         echo "ERROR! \"${REQUIRED_TOOLS_DARWIN}\" is missing."
         echo "  ${0} requires it to operate."
         echo "  Please install the needed tools with:"
         echo "  sudo port install coreutils gsed gawk snarf lynx"
         exit $ERROR_TOOLNOTFOUND
      fi
   done

   # will fix german utf-coding problems

   export LC_CTYPE=de_DE.ISO8859-15
   export LC_MESSAGES=en_US.ISO8859-15

   GNU_TEE=gtee		# coreutils
   GNU_RM=grm		# coreutils
   GNU_CAT=gcat		# coreutils
   GNU_WC=gwc		# coreutils
   GNU_TR=gtr		# coreutils
   GNU_DATE=gdate	# coreutils
   GNU_CUT=gcut		# coreutils
   GNU_EXPR=gexpr	# coreutils
   GNU_TAC=gtac		# coreutils
   GNU_SED=gsed		# gsed

   # additional needed:
   # gawk snarf lynx

else
   ### all other unix ###################################

   for REQUIRED_TOOL in ${REQUIRED_TOOLS}
   do
	  which ${REQUIRED_TOOL} >/dev/null 2>&1
	  if [ $? -eq 1 ]; then
	       echo "ERROR! \"${REQUIRED_TOOL}\" is missing. ${0} requires it to operate."
	       echo "Please install \"${REQUIRED_TOOL}\"."
	       exit $ERROR_TOOLNOTFOUND
	  fi
   done
fi

# create needed directories

if [ ! -d channels ]; then
   mkdir channels
else
   rm -f channels/* > /dev/null 2>&1
fi

if [ ! -d data ]; then
   mkdir data
else
   rm -f data/* > /dev/null 2>&1
fi

if [ ! -d epg ]; then
   mkdir epg
else
   rm -f epg/* > /dev/null 2>&1
fi

if [ ! -d icons ]; then
   mkdir icons
fi

if [ ! -d links ]; then
   mkdir links
else
   rm -f links/* > /dev/null 2>&1
fi

if [ ! -d xml ]; then
   mkdir -p xml/raw
else
   rm -f xml/epg.xml > /dev/null 2>&1
   rm -f xml/raw/* > /dev/null 2>&1
fi



# In the case the setup-file is not allready local we
# download the file from the project url

if [ -e setup-teleboy.ch ]; then

   # The setup-file do exist but as long all channels are
   # disabled we do not contiune .....
   # edit the file and remove # on a wished channel ....

   ccounter=$(grep -v "#" setup-teleboy.ch  | $GNU_WC -l)

   if [ "$ccounter" == "0" ] ; then
      clear
      echo
      echo there are no channels to fetch definied.
      echo Remove [#] on every channel that you wish to fetch.
      echo Please edit the file setup-teleboy.ch
      echo
      exit $ERRRO_EDIT_CONFIGURATION
   fi

else
   snarf -m -q  http://epg-swiss.googlecode.com/files/setup-0.9O setup-teleboy.ch
   if [ -e setup-teleboy.ch ]; then
      clear
      echo "The configuration file for this grabber is now local installed."
      echo
      echo "Please edit the file setup-teleboy.ch with a editor of your choice."
      echo "Do not add or remove lines from this file !!!!!!"
      echo
      echo "The script do exit now, that you can customize the channels to collect."
      echo
      echo
      exit $ERRRO_EDIT_CONFIGURATION

  else
      clear
      echo "The configuration file for this grabber can not be downloaded."
      echo
      echo "Please check on the project url that you are using the latest version."
      echo
      exit $ERROR_TOOLCHAIN_TO_OLD
  fi
fi

#########################################################



#########################################################
#             Check-Arguments                           #
#########################################################
if [ $# -lt $nargs ]; then
  clear
  echo
  echo ----------------------------------------------------------------------------
  SCRIPT=$(basename $0)
  echo "script    :" $SCRIPT
  $GNU_CAT VERSION
  echo "copyright : (C) <2010>  <linuxluemmel.ch@gmail.com> <thomas.oeding@gmail.com>"
  cd "$SCRIPTDIR" && echo changed to $SCRIPTDIR
  echo ----------------------------------------------------------------------------
  echo
  echo example of usage:
  echo
  echo ./epg-swiss.sh slow 4
  echo
  echo will donwload complet epg-data for 5 days
  echo the index is starting with 0
  echo
  echo ./epg-swiss fast 1
  echo
  echo will donwload only minimal epg-data for 2 days
  echo the index is starting with 0
  echo
  exit $ERROR_WRONG_ARG
fi
#########################################################



#########################################################
#             Show Banner                               #
#########################################################
if [ $1 = "slow"  ] ;  then
   clear
   echo
   echo ----------------------------------------------------------------------------
   SCRIPT=$(basename $0)
   echo "script    :" $SCRIPT
   $GNU_CAT VERSION
   echo "epg-data  : [complet]"
   echo "copyright : (C) <2010>  <linuxluemmel.ch@gmail.com>  <thomas.oeding@gmail.com>"

   cd "$SCRIPTDIR" && echo changed to $SCRIPTDIR
   echo ----------------------------------------------------------------------------
else
   if [ $1 = "fast"  ] ;  then
        clear
        echo
        echo ----------------------------------------------------------------------------
        SCRIPT=$(basename $0)
        echo "script    :" $SCRIPT
        $GNU_CAT VERSION
        echo "epg-data  : [showname only]"
        echo "copyright : (C) <2010>  <linuxluemmel.ch@gmail.com>  <thomas.oeding@gmail.com>"
        cd "$SCRIPTDIR" && echo changed to $SCRIPTDIR
        echo ----------------------------------------------------------------------------
   else
        clear
        echo
        echo ----------------------------------------------------------------------------
        SCRIPT=$(basename $0)
        echo "script    :" $SCRIPT
        $GNU_CAT VERSION
        echo "copyright : (C) <2010>  <linuxluemmel.ch@gmail.com>  <thomas.oeding@gmail.com>"
        cd "$SCRIPTDIR" && echo changed to $SCRIPTDIR
        echo ----------------------------------------------------------------------------
        echo
        echo wrong parameter for mode :  only slow or fast are supported ...
        echo
        echo example:
        echo
        echo ./epg-swiss.sh slow 4 or ./epg-swiss fast 1
        echo
        exit $ERROR_AWRONG
   fi
fi
#########################################################




#########################################################
#             Check day index                           #
#########################################################
if [ $2 -gt $MAXDAYS ] ; then
   echo
   echo parameter index is to high : 0-12 is valid
   echo
   echo You can increase this values with the variable MAXDAYS inside this script.
   echo It is not recommendet to use a higher value than 12 !
   echo
   exit $ERROR_INDEX_TO_HIGH
fi


#########################################################
#             Check-Host                                #
#########################################################

check_host=$(ping -c1 www.teleboy.ch)
retval=$?
if [ $retval -eq 2 ] ; then
   echo
   echo host www.teleboy.ch is not reachable ....
   echo please try later to run this script or check your
   echo internet connection.
   echo
   exit $ERROR_HOST
fi
#########################################################



#########################################################
#             Calculate Offeset of Days                 #
#########################################################

# If the script is running between 00:00 and 06.00 we have to
# set the the start date one day before

exectime=$(TZ=Europe/Zurich $GNU_DATE +%H)
datefix=0

if [ "$exectime" -lt "06" ] ; then
   datefix=1
fi

syear=$(TZ=Europe/Zurich $GNU_DATE +%Y --date="-$datefix days")
smonth=$(TZ=Europe/Zurich $GNU_DATE +%m --date="-$datefix days")
sday=$(TZ=Europe/Zurich $GNU_DATE +%d --date="-$datefix days")

eyear=$($GNU_DATE --date="+$2 days" "+%Y%m%d" | $GNU_CUT -b 1-4)
emonth=$($GNU_DATE --date="+$2 days" "+%Y%m%d" | $GNU_CUT -b 5-6)
eday=$($GNU_DATE --date="+$2 days" "+%Y%m%d" | $GNU_CUT -b 7-8)
exectime=$($GNU_DATE)

homedir=$(pwd)

if [ $retval -eq 0 ] ; then
   echo
   echo "host          [www.teleboy.ch] is running"
   echo "execution    " [$exectime]
   echo "start-date   " [$syear-$smonth-$sday]
   echo "end-date     " [$eyear-$emonth-$eday]

if [ $datefix -eq 0 ] ; then
   echo "date-fix     " [inactive]
else
   echo "date-fix     " [active]
fi
   echo
fi

#########################################################




#########################################################
#             Extract Data (Fast-Mode)                  #
#########################################################
if [ $1 = "fast"  ] ;  then

   echo This mode is not supported in this release.
   echo A later release may include again the fast-mode
   echo The script do exit now.
   echo
   exit $ERROR_FAST_MODE
fi
#########################################################



#########################################################
#             Download Data                             #
#########################################################
baseurl="http://www.teleboy.ch/"
plimit=45
index=0

cd data
$GNU_RM epg* >/dev/null 2>&1
cd ../epg
$GNU_RM * >/dev/null 2>&1
$GNU_RM ../xml/*.xml >/dev/null 2>&1
cd ../data
echo "download data [www.teleboy.ch] is running"
echo

if [ "$SYSTEM" = "Darwin" ]; then
   echo '<?xml version="1.0" encoding="ISO8859-15"?>' > ../xml/epg.xml
else
   echo '<?xml version="1.0" encoding="UTF-8"?>' > ../xml/epg.xml
fi

echo '<!DOCTYPE tv SYSTEM "xmltv.dtd">' >> ../xml/epg.xml
echo '<tv source-info-url="http://tv.teleboy.ch/" source-data-url="http://tv.teleboy.ch/programm/" generator-info-name="XMLTV" generator-info-url="http://code.google.com/p/epg-swiss/">' >> ../xml/epg.xml
echo "<!-- version           "[0.9M]" -->" >> ../xml/epg.xml
echo "<!-- execution         "[$exectime]" -->" >> ../xml/epg.xml
echo "<!-- start             "[$syear-$smonth-$sday]" -->" >> ../xml/epg.xml
echo "<!-- end               "[$eyear-$emonth-$eday]" -->" >> ../xml/epg.xml
echo  >> ../xml/epg.xml
echo  >> ../xml/epg.xml

# All lines from the setup-file are read line by line
# If a line starts with '#' then the channel will not be downloaded ...

while read line
do
  index=`$GNU_EXPR $index + 1`
  data_channel=$(echo $line | grep "#")
  retval=$?
  mod=$(($index % $plimit))

  # This channel must be downloaded because no '#' is found inside the settings

  if [ $retval -eq 1 ] ; then
     idc=$(echo $line | $GNU_AWK '{print $1}')
     icon=$(echo $line | $GNU_AWK '{print $2}')

     if [ "$SHOWINDEX" == "TRUE" ] ; then
        echo "download data [$idc $index] "
     else
        echo -n "download data [$idc $index] "
     fi

     echo '<channel id="'$index'.www.teleboy.ch">' >> ../xml/epg.xml
     echo '         <display-name lang="de">'$idc'</display-name>' >> ../xml/epg.xml

     # On request the file-location of a channel-icon is included

     if [ "$INCLUDE_ICONS" == "TRUE" ] ; then
        if [ "$icon" == "none" ] ; then
           echo no icon for this channel > /dev/null
        else
           echo '         <icon src="'$icon'" /> ' >> ../xml/epg.xml
        fi
     fi

     echo '</channel>' >> ../xml/epg.xml

     day_index=0
     if [ "$SHOWINDEX" == "TRUE" ] ; then
         echo all the next url-locations will be donwloaded ...
         echo
     else
         echo -n [
     fi

     for (( ; ; ))
     do
       if [ $day_index -gt $2 ] ; then
          if [ "$SHOWINDEX" == "TRUE" ] ; then
             echo
          else
              echo ]
          fi
          break
       else
          ayear=$($GNU_DATE --date="+$day_index days" "+%Y%m%d" | $GNU_CUT -b 1-4)
          amonth=$($GNU_DATE --date="+$day_index days" "+%Y%m%d" | $GNU_CUT -b 5-6)
          aday=$($GNU_DATE --date="+$day_index days" "+%Y%m%d" | $GNU_CUT -b 7-8)
          data=$(echo "http://www.teleboy.ch/programm/stationview.php?date="$ayear-$amonth-$aday"&station_id="$index"&time=00:00")
          outfile=$(echo "epg"$index"_"$day_index".html")
          snarf -m -q "$data" $outfile 2>/dev/null &
          sleep 0.2
          if [ "$SHOWINDEX" == "TRUE" ] ; then
             echo [$data] [$outfile]
          else
             echo -n \#
          fi
       fi
       day_index=`$GNU_EXPR $day_index + 1`
     done
     sleep 0.3
  fi
done < ../setup-teleboy.ch
echo  >> ../xml/epg.xml
echo  >> ../xml/epg.xml
cd ..
echo
echo download data [www.teleboy.ch] is passed
echo "waiting       [last snarf process]"


# We wait until all snarf processes are finished ...

LOOP=1
while [ $LOOP -eq '1'  ];
do
   COUNTER=$(ps axu | grep snarf | grep -v grep | $GNU_WC -l )
   if [ "$COUNTER" == "0" ] ; then
      LOOP=0
      echo
   else
      LOOP=1
   fi
done

echo

#########################################################




#########################################################
#             Extract Links (Slow-Mode)                 #
#########################################################

echo
echo "extract links [www.teleboy.ch] is running"
echo

if [ "$VERBOSE" == "TRUE" ] ; then
    echo "all links do follow now "
    echo
else
    echo -n [
fi

$GNU_RM epg/epglinks.tvtv >/dev/null 2>&1
$GNU_RM epg/epgchannels.tvtv >/dev/null 2>&1
$GNU_RM epg/greplist >/dev/null 2>&1

cd data

# we need the exact order ..... or the xml do not contain the right order
channels=$(ls -lv * |  $GNU_SED 's/^.* //' > channels_list)

index=0

# In this main-loop we do extract all external links to epg-data
# because we are running in slow-mode

while read line
do
  channel_index=$(basename $line html | $GNU_SED 's/\./ /g' | $GNU_SED 's/epg/ /g' | $GNU_SED  's/_/ /g' | $GNU_AWK '{print $1}')
  index=`$GNU_EXPR $index + 1`

  # Search links and store them inside epglinks

  grep href $line | grep programm/show | grep clue_image | $GNU_TR '"' ' ' | $GNU_AWK '{print $3}' |  $GNU_TEE -a ../epg/greplist >> ../epg/epglinks.tvtv
  epglinks=$($GNU_CAT  ../epg/greplist | $GNU_WC -l)
  if [ $epglinks -gt 0 ] ; then

       if [ "$VERBOSE" == "TRUE" ] ; then
          echo "extract link  [$line $channel_index $epglinks ] "
       else
          echo -n \#
       fi

       y=1
       let z=epglinks+1
       while [  $y -lt $z ]; do
             echo $channel_index >> ../epg/epgchannels.tvtv
             let y=y+1
       done
  fi

  # Cleanup

  $GNU_RM ../epg/greplist > /dev/null 2>&1
  $GNU_RM $line >/dev/null 2>&1

done < channels_list
$GNU_RM channels >/dev/null 2>&1

if [ "$VERBOSE" == "TRUE" ] ; then
    echo " "
else
    echo -n ]
fi

echo
echo
echo "extract links [www.teleboy.ch] is passed"
cd ..

# We should have now a few links or something is realy wrong ....


if [ -f epg/epgchannels.tvtv ] ; then

   LINK_COUNTER=$($GNU_WC -l epg/epgchannels.tvtv | $GNU_AWK '{print $1}')
   echo "extract links [links found=$LINK_COUNTER]"
   echo
else
   clear
   echo
   echo
   echo No links were found ....
   echo
   echo This should realy not happen the script do exit now ...
   echo
   echo - Please use newer versions of the tools used inside this script
   echo - Are you not using a UTF-8 terminal ?
   echo - Update your system to a newer system
   echo - Please update the grep-line to search for the links inside the script
   echo
   exit $ERROR_TOOLCHAIN_TO_OLD
fi

sleep 1
#########################################################






#########################################################
#             Mirror Links                              #
#########################################################
echo
echo "mirror links  [www.teleboy.ch] is running"
LINK_COUNTER=$($GNU_WC -l epg/epglinks.tvtv | $GNU_AWK '{print $1}')
echo "mirror links  [$LINK_COUNTER]"
echo
echo -n [\#
cd epg
plimit=50
index=0

while read line
do
     index=`$GNU_EXPR $index + 1`
     indexwithzeros=`$GNU_EXPR "00000$index" : '.*\(.....\)$'`
     mod=$(($index % $plimit))

     # We are nice and wait a few secounds

     if [ $mod -eq $compare ] ; then
          echo -n \#
          sleep 7
     fi
     data=$(echo "http://www.teleboy.ch"$line)
     outfile=$(echo "epg-"$indexwithzeros".html")
     $GNU_RM outfile > /dev/null 2>&1

     snarf -m -q "$data" $outfile 2>/dev/null &
     sleep 0.3
done < epglinks.tvtv
echo -n ]
echo
echo
echo "mirror links  [www.teleboy.ch] is passed"
echo "last index    [$index]"


# We wait until all snarf processes are finished  ...

echo "waiting       [last snarf process]"
LOOP=1
while [ $LOOP -eq '1'  ];
do
   COUNTER=$(ps axu | grep snarf | grep -v grep | $GNU_WC -l )
   if [ "$COUNTER" == "0" ] ; then
      LOOP=0
      echo
   else
      LOOP=1
   fi
done

echo
cd ..
##########################################################



##########################################################
#             Extract complet Data                       #
##########################################################
rm -rf  xml/raw/* > /dev/null 2>&1
mkdir -p xml/raw # create if not exists (bugfix)
cd epg
echo
echo "export        [xml] is running"


# we need the exact order ..... or the xml do not contain the right order

channelsvar=$(ls -lv epg-* |  $GNU_SED 's/^.* //' > channels)

channelsvar=""
EPG_COUNTER=$($GNU_WC -l channels | $GNU_AWK '{print $1}')
echo "extract shows [$EPG_COUNTER]"
echo
echo -n [\#

index=0
plimit=80
y=0
zerozero="00"
while read line
do

  index=`$GNU_EXPR $index + 1`
  indexwithzeros=`$GNU_EXPR "00000$index" : '.*\(.....\)$'`

  channel_id_num=$(head -$index epgchannels.tvtv | tail -1)
  channel_id=$channel_id_num'.www.teleboy.ch'

  # Never try to build your own html-parser with sed and awk
  # This would end in a desaster .... use lynx and dumpe the website

  lynx -dump $line > tmp.html
  cat tmp.html  > ../xml/raw/$line

  # Inside the variable line_1 we have start line
  # line_2 contains the last line with the data

  line_1=$(grep -hn  '\[119\]' tmp.html | $GNU_TR ':' ' ' | $GNU_AWK '{print $1}')
  line_2=$(grep -hn  'AGB' tmp.html | $GNU_TR ':' ' ' | $GNU_AWK '{print $1}')

  $GNU_CAT tmp.html  | $GNU_SED $line_1,$line_2!d  > tmp-data

  $GNU_CAT tmp-data | $GNU_SED -e 's/^[ \t]*//' | $GNU_SED -e '/weitersagen/d' |  $GNU_SED '/^$/d' | \
  grep -v "I-Frames " | grep -v "( ) Alle Sender" | grep -v "(_) Alle Sender (_) Meine Sender" | grep -v "Keine TV-Sendungen zu" | \
  grep -v "IFRAME:"  | grep -v "Erweitern Sie Ihre Senderauswahl" | grep -v "^Sender" | \
  grep -v "^schauen$" | grep -v "Sendung schauen$"  | grep -v "^aufnehmen" | \
  grep -v "^WebTV" | grep -v "^pps" | grep -v "^setzen" > tmp-data2


  grep "Sie haben die Sendung leider" tmp.html
  retval=$?
  if [ $retval -eq 1 ] ; then

     # we extract show-name /  date / start-time / end-time

     dateline=$(grep "[0-9][0-9][0-9][0-9]   \| [0-9][0-9]:[0-9][0-9] - [0-9][0-9]:[0-9][0-9] Uhr$" tmp-data2)

     date1=$(echo $dateline | $GNU_AWK '{print $2}' | $GNU_TR -d "." )
     time1=$(echo $dateline | $GNU_AWK '{print $4}' | $GNU_TR -d ":" )
     time2=$(echo $dateline | $GNU_AWK '{print $6}' | $GNU_TR -d ":" )

     # Extract name of show
     # Most of the shows neeed only a single line for the title ...

     tline1=$($GNU_CAT tmp-data2 | grep -v "^\[" |  head -1  | $GNU_SED 's/&/&amp;/g' | $GNU_SED 's/*/\&#42;/g')
     stline=$($GNU_CAT tmp-data2 | grep -v "^\[" |  head -3  | $GNU_SED 's/&/&amp;/g' | $GNU_SED 's/*/\&#42;/g' | tail -1 | grep "Auf Watchlist setzen")

     if [ -n "$stline" ] ; then
        tline2=''
     else
        tline2=$($GNU_CAT tmp-data2 | grep -v "^\[" |  head -2 | tail -1 | $GNU_SED 's/&/&amp;/g' | $GNU_SED 's/*/\&#42;/g')
     fi

     year1=$(echo $date1 | $GNU_CUT -b 5-8)
     month1=$(echo $date1 | $GNU_CUT -b 3-4)
     day1=$(echo $date1 | $GNU_CUT -b 1-2)

     time_1=$(echo $time1 | $GNU_SED "s/:/ /g" | $GNU_SED "s/ //g")
     time_2=$(echo $time2 | $GNU_SED "s/:/ /g" | $GNU_SED "s/ //g")

     # Set program date - program with a start time before 06:00 is part of the next day

     if [ "$time_1" -lt "0600" ]; then
        programdate1=`TZ=Europe/Zurich $GNU_DATE +%Y%m%d --date="$year1$month1$day1 +1 day"`
        filler1=`TZ=Europe/Zurich $GNU_DATE +%z --date="$year1$month1$day1 +1 day"`
     else
        programdate1=`TZ=Europe/Zurich $GNU_DATE +%Y%m%d --date=$year1$month1$day1`
        filler1=`TZ=Europe/Zurich $GNU_DATE +%z --date="$year1$month1$day1"`
     fi

     if [ "$time_2" -lt "0600" ]; then
        programdate2=`TZ=Europe/Zurich $GNU_DATE +%Y%m%d --date="$year1$month1$day1 +1 day"`
        filler2=`TZ=Europe/Zurich $GNU_DATE +%z --date="$year1$month1$day1 +1 day"`
     else
        programdate2=`TZ=Europe/Zurich $GNU_DATE +%Y%m%d --date=$year1$month1$day1`
        filler2=`TZ=Europe/Zurich $GNU_DATE +%z --date="$year1$month1$day1"`
     fi

     description=$($GNU_CAT tmp-data2 | grep -v "Watchlist setzen" | grep -v "Wiederholung" | grep -v "^Laufzeit" | grep -v ", \[" | \
     grep -v "Stereo" | grep -v "^ogramm$" | grep -v "Diese Sendung in einem Bookmark-Dienst speichern" | grep -v "Televisio)" | \
     grep -v "16:9" | grep -v "Moderation:" | grep -v "Buch" | grep -v "Regie" | grep -v "^\[[0-9]" | grep -v ", [0-9][0-9][0-9][0-9]$" | grep -v "^O-Titel" | \
     grep -v "^[A-Z][a-z], [0-9][0-9].[0-9][0-9].[0-9][0-9][0-9][0-9]$" | grep -v "TXT-Untertitelt" | grep -v "(Vom" | grep -v "^Mit:" |   \
     grep -v "^[0-9][0-9].[0-9][0-9]." | grep -v "^[0-9][0-9][0-9][0-9]$" |   grep -v "Dolby Surround" |  grep -v "gefunden.$" | \
     grep -v "Schwarz-Weiss" |  grep -v "^TV-Programm" | grep -v "^D$"  | $GNU_AWK 'NR>2' | grep -v "$dateline" | grep -v "T)$" | grep -v "Corporation)$" | \
     grep -v "Inc. )$" | $GNU_SED 's/*/\&#42;/g' | $GNU_SED -e 's/\n*//' |  $GNU_TEE tmp-data4)

     # In the case that a secound title line was used we need to remove this line ...

     if [ -n "$tline2" ] ; then
        if [ -n "$description" ] ; then
           $GNU_CAT tmp-data4 | grep -v "$tline2" > tmp-data3
           cp tmp-data3 tmp-data4
        fi
     fi

     if [ -n "$description" ] ; then
        last_line=$(tail -1 tmp-data4 | grep ")$")
        if [ -n "$last_line" ] ; then
           $GNU_CAT tmp-data4 | grep -v "$last_line" > tmp-data3
           cp tmp-data3 tmp-data4
        fi
     fi

     genre=$($GNU_CAT tmp-data | grep -A 2  "$dateline" | tail -1 | $GNU_SED 's/Auf Watchlist setzen\/entfernen/ /g' | \
     $GNU_SED 's/\[121\]/ /g' | $GNU_SED -e 's/^[ \t]*//')

     grep "16:9" tmp-data2 > /dev/null 2>&1
     retval=$?
     if [ $retval -eq 0 ] ; then
         video="16:9"
     else
         video=""
     fi

     grep "Schwarz-Weiss" tmp-data2 > /dev/null 2>&1
     retval=$?
     if [ $retval -eq 0 ] ; then
         bvideo="true"
     else
         bvideo=""
     fi

     grep "TXT-Untertitelt" tmp-data2 > /dev/null 2>&1
     retval=$?
     if [ $retval -eq 0 ] ; then
        teletext="true"
     else
        teletext=""
     fi

     grep "Stereo" tmp-data2 > /dev/null 2>&1
     retval=$?
     if [ $retval -eq 0 ] ; then
        audio="true"
     else
        audio=""
     fi

     grep "Dolby Surround" tmp-data2 > /dev/null 2>&1
     retval=$?
     if [ $retval -eq 0 ] ; then
        saudio="true"
     else
        saudio=""
     fi

     grep "vom [0-9]" tmp-data2 > /dev/null 2>&1
     retval=$?
     if [ $retval -eq 0 ] ; then
        rerun="true"
     else
        rerun=""
     fi

     grep "^Mit:" tmp-data2 > /dev/null 2>&1
     retval=$?
     if [ $retval -eq 0 ] ; then
        actors=$(grep -A 5 "^Mit:" tmp-data2 | grep -v "Stereo" | grep -v "TV-Programm" |  grep -v "16:9" | grep -v "^Buch" | \
        grep -v "Kamera" | grep -v "Musik" |  grep -v "Wiederholung" | grep -v "^TXT-Untertitelt" | grep -v "http" | grep -v "Schwarz-Weiss" | \
        grep -v "Sendung aufnehmen" | grep -v "ADTECH" | grep -v "[0-9][0-9].[0-9][0-9]." | grep -v "^ogramm$" | \
        $GNU_SED 's/Mit:/ /g' |  $GNU_SED 's/\[/ /g' | $GNU_SED 's/\]/ /g' |  $GNU_TR -d [:digit:] | $GNU_AWK '$1=$1' | $GNU_SED 's/&/&amp;/g' | \
        $GNU_SED 's/*/\&#42;/g' | $GNU_TEE tmp-data5)
     else
        actors=""
     fi
  fi

  # we export the data into xml

  echo '<programme start="'$programdate1$time_1$zerozero $filler1'" stop="'$programdate2$time_2$zerozero $filler2'" channel="'$channel_id_num'.www.teleboy.ch">' >> ../xml/epg.xml
  echo "     <!-- show-index"[$indexwithzeros]" -->" >> ../xml/epg.xml
  echo "     <!-- channel-index"[$channel_id_num.www.teleboy.ch]" -->" >> ../xml/epg.xml
  echo "     <!-- data-source"[$line]" -->" >> ../xml/epg.xml


  # Title of show

  if [ -n "$tline2" ] ; then
     echo '     <title lang="de">'$tline1 $tline2'</title>' >> ../xml/epg.xml
  else
     echo '     <title lang="de">'$tline1'</title>' >> ../xml/epg.xml
  fi

  # Description of show

  if [ -n "$description" ] ; then
     lines=$($GNU_WC -l tmp-data4 | $GNU_AWK '{print $1}')
     if [ $lines -gt "0" ] ; then
        echo '     <desc lang="de">' >> ../xml/epg.xml
        while read lineact
        do
          echo $lineact | $GNU_SED 's/&/&amp;/g' >> ../xml/epg.xml
        done < tmp-data4
        echo '     </desc>' >> ../xml/epg.xml
        description=''
    fi
  fi

  # Actors

  if [ -n "$actors" ] ; then
     lines=$(wc -l tmp-data5 | $GNU_AWK '{print $1}')
     if [ $lines -gt "0" ] ; then
          echo '     <credits>' >> ../xml/epg.xml
          while read lineact
          do
            echo '         <actor>'$lineact'</actor>' >> ../xml/epg.xml
          done < tmp-data5
          echo '     </credits>' >> ../xml/epg.xml
    fi
  fi

  # Genre

  if [ -n "$genre" ] ; then
     echo '     <category lang="de">'$genre'</category>' >> ../xml/epg.xml
  fi

  # Vide0

  if [ -n "$video" ] ; then
     echo '     <video><aspect>16:9</aspect></video>' >> ../xml/epg.xml
  fi

  # Audio

  if [ -n "$audio" ] ; then
     echo '     <audio><stereo>Stereo</stereo></audio>' >> ../xml/epg.xml
  fi

  # Rerun of the show

  if [ -n "$rerun" ] ; then
     echo '     <previously-shown />' >> ../xml/epg.xml
  fi

  # Teletext

  if [ -n "$teletext" ] ; then
     echo '     <subtitles type="teletext"/>' >> ../xml/epg.xml
  fi

  echo "</programme>" >> ../xml/epg.xml
  echo " " >> ../xml/epg.xml

  if [ "$VERBOSE" == "" ] ; then
     y=`$GNU_EXPR $y + 1`
     mod=$(($y % $plimit))
     if [ $mod -eq $compare ] ; then
        echo -n \#
     fi
  fi

  $GNU_RM $line >/dev/null 2>&1

  # Remove temp files

  if [ -e description ] ; then
     $GNU_RM description > /dev/null 2>&1
  fi

  if [ -e tmp.html ] ; then
     $GNU_RM tmp.html  >/dev/null 2>&1
  fi

  if [ -e tmp-data ] ; then
     $GNU_RM tmp-data  >/dev/null 2>&1
  fi

  if [ -e tmp-data2 ] ; then
     $GNU_RM tmp-data2 >/dev/null 2>&1
  fi

  if [ -e tmp-data3 ] ; then
     $GNU_RM tmp-data3 >/dev/null 2>&1
  fi

  if [ -e tmp-data4 ] ; then
     $GNU_RM tmp-data4 >/dev/null 2>&1
  fi

  if [ -e tmp-data5 ] ; then
     $GNU_RM tmp-data5 >/dev/null 2>&1
  fi


done < channels
$GNU_RM channels >/dev/null 2>&1

#  create eof inside xml

echo '</tv>' >> ../xml/epg.xml
echo -n ]
echo
echo
echo "export        [xml] is passed"
echo

# Cleanup directorys we have finished the job

$GNU_RM -f ../channels/* > /dev/null 2>&1
echo
echo directory cleaned [channels]

$GNU_RM -f  ../data/* > /dev/null 2>&1
echo directory cleaned [data]

$GNU_RM  ../epg/* > /dev/null 2>&1
echo directory cleaned [epg]

$GNU_RM -f ../links/* > /dev/null 2>&1
echo directory cleaned [links]
echo

echo ----------------------------------------------------------------------------
echo "script is finished and do exit now ..."
echo ----------------------------------------------------------------------------
echo
exit $ZERO_ERRORS
